﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Management.Automation;
using System.Management;
using System.Threading;
using Microsoft.PowerShell;
using System.IO;

namespace pshcli
{

    [Cmdlet("Exec", "Dwarfs")]
    public class DwarfBench : Cmdlet
    {

        public int override_proccount;
        public int override_threadcount;
        public bool override_xperftrace = false;
        public bool override_mpitrace = false;

        public bool _plotexcel = false;
        public bool _plotvampir = false;
        public bool _plotjumpshot = false;
        public bool _plotxperf = false;
        public static string _schedule_res = "";

        public bool runoncluster = false;

        private Object[] filternames;
        private Object[] filtersizes;
        private Object[] filterplatform;
        private Object[] filterparallel;

        public static GlobalSettings gsettings;
        public static Log glog;
        public static bool help = false;
        public static bool whatif = false;
        private bool isalive = false;



        #region PARAMETERS

        [Parameter(
            Mandatory = false,
            Position = 0,
            ValueFromPipelineByPropertyName = true,
            HelpMessage = "Specify a comma-separated list of dwarfs' names to be executed.")]
        public Object[] Names
        {
            set { filternames = value; }
        }

        [Parameter(
            Mandatory = false,
            Position = 1,
            ValueFromPipelineByPropertyName = true,
            HelpMessage = "Specify the kernel target platform. (Values: managed/unmanaged)")]
        public Object[] Platform
        {
            set { filterplatform = value; }
        }

        [Parameter(
            Mandatory = false,
            Position = 2,
            ValueFromPipelineByPropertyName = true,
            HelpMessage = "Specify parallel model to be used during execution. (Values: serial, omp, tpl, mpi, hybrid)")]
        public Object[] Parallel
        {
            set { filterparallel = value; }
        }

        [Parameter(
            Mandatory = false,
            Position = 3,
            ValueFromPipelineByPropertyName = true,
            HelpMessage = "Specify the input file to be used for execution. (Values: small, medium, large)")]
        public Object[] Input
        {
            set { filtersizes = value; }
        }

        [Parameter(
            Mandatory = false,
            HelpMessage = "Displays the help message.")]
        public SwitchParameter Help
        {
            set { help = true; }
        }

        [Parameter(
            Mandatory = false,
            HelpMessage = "Displays configuration, but does not execute.")]
        public SwitchParameter WhatIf
        {
            set { whatif = true; }
        }

        [Parameter(
            Mandatory = false,
            HelpMessage = "Thread count.")]
        public int ThreadCount
        {
            set { override_threadcount = value; }
        }


        [Parameter(
            Mandatory = false,
            HelpMessage = "MPI proc count.")]
        public int MpiProcCount
        {
            set { override_proccount = value; }
        }

        [Parameter(
            Mandatory = false,
            HelpMessage = "Enables xperf tracing.")]
        public SwitchParameter XperfTrace
        {
            set { override_xperftrace = true; }
        }

        [Parameter(
            Mandatory = false,
            HelpMessage = "Enables Mpi tracing.")]
        public SwitchParameter MpiTrace
        {
            set { override_mpitrace = true; }
        }

        [Parameter(
            Mandatory = false,
            HelpMessage = "Plot results to Excel.")]
        public SwitchParameter PlotExcel
        {
            set { _plotexcel = true; }
        }

        [Parameter(
          Mandatory = false,
          HelpMessage = "Plot results to Vampir.")]
        public SwitchParameter PlotVampir
        {
            set { _plotvampir = true; }
        }

        [Parameter(
            Mandatory = false,
            HelpMessage = "Plot results to JumpShot.")]
        public SwitchParameter PlotJumpShot
        {
            set { _plotjumpshot = true; }
        }

        [Parameter(
            Mandatory = false,
            HelpMessage = "Plot results to Xperfview.")]
        public SwitchParameter PlotXperf
        {
            set { _plotxperf = true; }
        }

        [Parameter(
            Mandatory = false,
            HelpMessage = "Specify whether to use a HPC cluster for execution.")]
        public SwitchParameter Cluster
        {
            set { runoncluster = true; }
        }

        [Parameter(
            Mandatory = false,
            HelpMessage = "Specify resource allocation unit")]
        public String AskedResource
        {
            set { _schedule_res = value; }
        }

        #endregion



        protected override void BeginProcessing()
        {
            string dwarfdir = System.Environment.GetEnvironmentVariable("DWARF_DIR");
            if (String.IsNullOrEmpty(dwarfdir))
                throw new Exception("ERROR: Environment variable DWARF_DIR not set. Terminating...");
            dwarfdir = dwarfdir.Trim();

            string logfile = dwarfdir + "Logs\\DwarfBenchLog.txt";
            glog = new Log(logfile);

            glog.WriteLog("BEGIN_PROCESSING", "Starting logging for session " + System.DateTime.Today.ToShortDateString() + "," + System.DateTime.Today.ToLocalTime() + ".");
            glog.WriteLog("BEGIN_PROCESSING", "Dwarf home directory is " + dwarfdir);


            Microsoft.Win32.SessionEndingEventHandler shandler = new Microsoft.Win32.SessionEndingEventHandler(SessionEndHandler);

            glog.WriteLog("BEGIN_PROCESSING", "Termination handler initialized.");


            isalive = true;
            Console.TreatControlCAsInput = false;
            glog.WriteLog("BEGIN_PROCESSING", "Registered console event handler.");

            try
            {
                gsettings = new GlobalSettings(dwarfdir);
            }
            catch (Exception gsettingsexp)
            {
                glog.WriteLog("BEGIN_PROCESSING", "Error generating global settings.");
                glog.WriteLog("BEGIN_PROCESSING", "Please make sure setup.bat has been executed prior to running this cmdlet.");
                glog.WriteLog("BEGIN_PROCESSING", gsettingsexp.Message);
                glog.WriteLog("BEGIN_PROCESSING", gsettingsexp.Source);
                glog.WriteLog("BEGIN_PROCESSING", gsettingsexp.StackTrace);
                glog.WriteLog("BEGIN_PROCESSING", gsettingsexp.TargetSite.Name);

            }

            all = new ProgressRecord(1, "Total dwarf execution", "Progress...");
            all.PercentComplete = 0;
            terminate = false;
            xperfon = false;


        }

        protected override void ProcessRecord()
        {

        }

        protected override void EndProcessing()
        {
            if (gsettings == null)
            {
                Console.WriteLine("A fatal error occured while reading global settings.");
                Console.WriteLine("Please see log file for detailed information.");
                Console.WriteLine("Terminating DwarfBench");

                whatif = false;
                help = false;
                isalive = false;
                glog.Close();
                return;
            }

            if (help)
            {
                DisplayHelpMessage();
                help = false;
                glog.WriteLog("DB_WF", "Help message displayed. Aborting here.");
                glog.Close();
                return;
            }

            try
            {
                List<Dwarf2> dwarfs = GetDwarfs3();
                Console.Write("\n\n");
                Console.WriteLine("Executing DwarfBench (v. 0.9.1)");
                Console.WriteLine("==============================================");
                Console.WriteLine("Published 2010 under Microsoft Public License.");
                Console.WriteLine("");
                Console.WriteLine("Logging enabled. Please see ({0}) for detailed info and debug information.", glog.lfile);
                Console.WriteLine("");
                Console.WriteLine("Execution configuration details: ");
                Console.WriteLine("     Xml config file: " + gsettings.xmlconfigfile);
                Console.WriteLine("     Thread count: " + gsettings.threadcount);
                Console.WriteLine("     Mpi count: " + gsettings.mpiproccount);
                if (runoncluster)
                {
                    Console.WriteLine("     HPC scheduler: " + gsettings.hpcscheduler);
                    gsettings.runoncluster = true;
                }
                else
                    gsettings.runoncluster = false;
                Console.WriteLine("     Execution on cluster enabled: " + runoncluster);
                if (!String.IsNullOrEmpty(_schedule_res))
                {
                    Console.WriteLine("     Level of resource allocation: " + _schedule_res);
                    gsettings.cluster_resource_alloc_level = _schedule_res;
                }
                Console.WriteLine("     Xperf tracing enabled: " + gsettings.xperftracingenabled);
                Console.WriteLine("     Mpi tracing enabled: " + gsettings.mpitracingenabled);
                Console.WriteLine("Execution directories:");
                Console.WriteLine("     Deployment directory: " + gsettings.deploymentdir);
                Console.WriteLine("     Dwarf directory: " + gsettings.dwarfdir);
                Console.WriteLine("     Cluster nodes deployment directory: " + gsettings.nodelocaltargetdir);
                Console.WriteLine("     Shared deployment directory: " + gsettings.sdeploymentdir);
                Console.WriteLine("");



                #region write filter to log
                glog.WriteLog("DB_WF", "Filtering dwarf configuration collection.");

                if (!(filternames == null || filternames.Length == 0))
                {
                    string _filternames = "";
                    foreach (string str in filternames)
                        _filternames += str + " ";
                    glog.WriteLog("DB_WF", "Filter for names: " + _filternames);
                }
                else
                    glog.WriteLog("DB_WF", "Filter for names not specified.");

                if (!(filterplatform == null || filterplatform.Length == 0))
                {
                    string _filterplatforms = "";
                    foreach (string str in filterplatform.ToArray())
                        _filterplatforms += str + " ";
                    glog.WriteLog("DB_WF", "Filter for platform: " + _filterplatforms);
                }
                else
                    glog.WriteLog("DB_WF", "Filter for platform not specified.");


                if (!(filterparallel == null || filterparallel.Length == 0))
                {
                    string _filterparallel = "";
                    foreach (string str in filterparallel)
                        _filterparallel += str + " ";
                    glog.WriteLog("DB_WF", "Filter for parallel: " + _filterparallel);
                }
                else
                    glog.WriteLog("DB_WF", "Filter for parallel not specified.");

                if (!(filtersizes == null || filtersizes.Length == 0))
                {
                    string _filtersizes = "";
                    foreach (string str in filtersizes)
                        _filtersizes += str + " ";
                    glog.WriteLog("DB_WF", "Filter for sizes: " + _filtersizes);
                }
                else
                    glog.WriteLog("DB_WF", "Filter for sizes not specified.");

                #endregion

                List<Dwarf2> filtered = new List<Dwarf2>();
                foreach (Dwarf2 __d in dwarfs)
                {
                    if (PassFilter(__d))
                        if (PassExclude(__d))
                        {
                            filtered.Add(__d);
                        }
                        else
                            Console.WriteLine("Dwarf excluded from run: " + __d.dwarfname + "." + __d.platform + "." + __d.parallel);
                }


                dwarfs = filtered;

                if (dwarfs.Count == 0)
                {
                    glog.WriteLog("DB_WF", "No dwarfs were selected.");
                    Console.WriteLine("\nNo dwarfs have met the filter criteria.\nPlease check spelling and/or exclude list in dwarfconfig.xml.");
                    Console.WriteLine("\nDwarfBench is exiting.");
                    whatif = false;
                    help = false;
                    isalive = false;
                    glog.Close();
                    return;
                }

                Console.WriteLine("The following dwarf configurations will be executed.");
                foreach (Dwarf2 __d in dwarfs)
                {
                    Console.WriteLine("Dwarfname: {0} - Platform:{1}, Parallel:{2}, Input:{3}.",
                        __d.dwarfname,
                        __d.platform,
                        __d.parallel,
                        __d.size);
                }
                if (dwarfs.Count > 150)
                    Console.WriteLine("Note: You have chosen a larger amound of configurations to execute which can result in long execution times. Please use filters to reduce the number of executions.\n");


                #region write filtered dwarfs to log
                Console.WriteLine("Total number of configured runs: " + dwarfs.Count);
                glog.WriteLog("DB_WF", "Total count of configured dwarf runs: " + dwarfs.Count);
                glog.WriteLog("DB_WF", "Writing filtered dwarf configuration collection.");
                foreach (Dwarf2 __d in dwarfs)
                {
                    glog.WriteLog("DB_WF", "dwarfname: " + __d.dwarfname);
                    glog.WriteLog("DB_WF", "   >platform: " + __d.platform);
                    glog.WriteLog("DB_WF", "   >parallel: " + __d.parallel);
                    glog.WriteLog("DB_WF", "   >input: " + __d.size);
                    glog.WriteLog("DB_WF", "   >workdir: " + __d.workdir);
                    glog.WriteLog("DB_WF", "   >inputfile: " + __d.inputfile);
                    glog.WriteLog("DB_WF", "   >resultdir: " + __d.resultdir);
                    glog.WriteLog("DB_WF", "   >resultfile: " + __d.resultfile);
                    glog.WriteLog("DB_WF", "   >profiledir: " + __d.profiledir);
                    glog.WriteLog("DB_WF", "   >profilefile: " + __d.profilefile);
                    glog.WriteLog("DB_WF", "   >numthreads: " + __d.numthreads);
                    glog.WriteLog("DB_WF", "   >scheduler: " + __d.scheduler);
                    glog.WriteLog("DB_WF", "   >jobstr: " + __d.jobstr);
                    glog.WriteLog("DB_WF", "   >mpienabled: " + __d.mpitrace);
                    glog.WriteLog("DB_WF", "   >mpinumprocs: " + __d.numprocs);
                    glog.WriteLog("DB_WF", "   >mpiexecstr: " + __d.mpiexecstr);
                    glog.WriteLog("DB_WF", "   >mpitracefile: " + __d.mpitracefile);
                    glog.WriteLog("DB_WF", "   >xperfenabled: " + __d.xperftrace);
                    glog.WriteLog("DB_WF", "   >xperftracefile: " + __d.xperftracefile);
                    //Execution
                }
                #endregion

                if (whatif)
                {
                    Console.WriteLine("WhatIf specified. Execution will be omitted.");
                    glog.WriteLog("DB_WF", "WhatIf. Aborting here.");
                    whatif = false;
                    help = false;
                    isalive = false;
                    glog.Close();
                    return;
                }


                //EXECUTE DWARF

                System.Diagnostics.Stopwatch allExec = new System.Diagnostics.Stopwatch();

                allExec.Start();
                Console.WriteLine("");
                glog.WriteLog("DB_WB", "Starting execution of dwarflist.");
                int failed = RunDwarfs(dwarfs);
                glog.WriteLog("DB_WB", "Dwarflist execution complete.");
                Console.WriteLine("Execution complete.");

                allExec.Stop();

                if (failed != 0)
                {
                    glog.WriteLog("DB_WF", "Number of failed dwarfs: " + failed);
                    Console.WriteLine("Number of failed dwarfs: " + failed);
                    Console.WriteLine(@"Please see log file for details. (.\Logs\DwarfBenchLog.txt)");
                }
                else
                {
                    glog.WriteLog("DB_WF", "Exit codes and job states indicate all dwarfs executed successfully.");
                }

                Console.WriteLine("DwarfBench total execution time: " + allExec.ElapsedMilliseconds / (double)1000 + " seconds.");

                //VISUALIZE
                if (!terminate)
                {

                    #region PLOT_EXCEL
                    if (_plotexcel)
                    {
                        glog.WriteLog("VIS", "Starting Excel visualisation");

                        if (!ExcelWriter.ExcelInstalled())
                        {
                            glog.WriteLog("VIS", "Excel not installed and/or cannot be initialized.");
                            Console.WriteLine("Error: Cannot access Excel executable (" + gsettings.excelpath + ").");
                            Console.WriteLine("Cannot view results in Excel. Skipping \"-PlotExcel\"");
                        }
                        else
                        {
                            bool createnewfile = false;
                            if (!System.IO.File.Exists(gsettings.excelresultfile))
                            {
                                createnewfile = true;
                                glog.WriteLog("VIS", "Creating new Excel file: " + gsettings.excelresultfile);
                            }

                            if (
                                (HelperMethods.IsFileInUse(gsettings.excelresultfile) ||
                                !HelperMethods.IsFileReadable(gsettings.excelresultfile, FileMode.Open, FileAccess.ReadWrite, FileShare.None
                                )) && !createnewfile)
                            {
                                glog.WriteLog("VIS", "Error, file in use or not usable: " + gsettings.excelresultfile);
                                Console.WriteLine("Error: Excel result file (" + gsettings.excelresultfile + ") is in use.");
                                Console.WriteLine("Cannot view results in Excel. Skipping \"-PlotExcel\"");
                            }
                            else
                            {
                                glog.WriteLog("VIS", "Writing dwarf logs to Excel file (" + gsettings.excelresultfile + ").");
                                Console.WriteLine("Starting Excel visualisation.");
                                Console.WriteLine("Writing results to excel file... ");
                                ExcelWriter.WriteResultsToExcelFile(gsettings.excelresultfile, gsettings, dwarfs, createnewfile);
                                Console.Write("Done.");

                                glog.WriteLog("VIS", "Starting Excel as separate process.");
                                System.Diagnostics.Process.Start(gsettings.excelresultfile);
                            }
                        }
                    }
                    #endregion

                    #region PLOT_XPERF
                    if (_plotxperf)
                    {
                        foreach (Dwarf2 _dwarf in dwarfs)
                        {
                            if (terminate)
                                break;

                            if (_dwarf.xperftrace && _dwarf.success)
                            {
                                Console.WriteLine("Starting Xperf visualisation.");
                                glog.WriteLog("VIS", "Starting Xperf visualisation");

                                glog.WriteLog("VIS", "Opening Xperfview for " +
                                    _dwarf.dwarfname + "[" +
                                    _dwarf.platform + "," +
                                    _dwarf.parallel + "," +
                                    _dwarf.size + "]" + ".");

                                try
                                {
                                    HelperMethods.ExecCommand("cmd.exe", "/C xperfview -i " + "\"" + _dwarf.xperftracefile + "\"", false);
                                }
                                catch (Exception xperfexp)
                                {
                                    Console.WriteLine("Error starting Xperfview for dwarf " + _dwarf.dwarfname);
                                    glog.WriteLog("VIS", "Error: Cannot start Xperfview: " + xperfexp.Message);
                                }
                            }
                            else
                            {
                                if (!_dwarf.success)
                                    Console.WriteLine("Skipping Xperfview for " + _dwarf.internalname.ToString() + " because kernel execution failed.");
                                if (!_dwarf.xperftrace)
                                    Console.WriteLine("Skipping Xperfview for " + _dwarf.internalname.ToString() + " because -XperfTrace was not enabled.");
                                glog.WriteLog("VIS", "Skipping Xperfview for " + _dwarf.internalname.ToString() + ", because xperf tracing was not enabled and/or dwarf failed.");
                            }
                        }

                    }
                    #endregion

                    #region PLOT_VAMPIR
                    if (_plotvampir)
                    {

                        if (!File.Exists(gsettings.vampirbin))
                        {
                            glog.WriteLog("VIS", "Error, cannot access file: " + gsettings.vampirbin);
                            Console.WriteLine("Error: Cannot access Vampir executable (" + gsettings.vampirbin + ").");
                            Console.WriteLine("Cannot view results in Vampir. Skipping \"-PlotVampir\"");
                        }
                        else
                        {
                            foreach (Dwarf2 _dwarf in dwarfs)
                            {
                                if (terminate)
                                    break;

                                if (_dwarf.success && _dwarf.mpitrace &&
                                    (
                                    _dwarf.parallel.Contains("mpi") ||
                                    _dwarf.parallel.Contains("mpinet") ||
                                    _dwarf.parallel.Contains("hybrid")
                                    )
                                    )
                                {
                                    Console.WriteLine("Starting Vampir visualisation.");
                                    glog.WriteLog("DB_WF", "Starting Vampir visualisation");
                                    glog.WriteLog("VIS", "Opening Vampir MPI trace analysis for " +
                                        _dwarf.dwarfname + "[" +
                                        _dwarf.platform + "," +
                                        _dwarf.parallel + "," +
                                        _dwarf.size + "]" + ".");

                                    string otffile = "";

                                    if (runoncluster && (!String.IsNullOrEmpty(gsettings.hpcscheduler)))
                                    {
                                        glog.WriteLog("VIS", "Opening Vampir for: " + _dwarf.otff);
                                        glog.WriteLog("VIS", "Vampir executable: " + gsettings.vampirbin);

                                        //launch vampir
                                        HelperMethods.ExecCommand(
                                            gsettings.vampirbin,
                                            _dwarf.otff,
                                            false);
                                    }
                                    else
                                    {
                                        //Process tracing information 
                                        string sdkpath = System.Environment.GetEnvironmentVariable("CCP_SDK");
                                        if (sdkpath != null) sdkpath = sdkpath.Trim(); else sdkpath = "";
                                        string mpicsync = sdkpath + "Bin\\mpicsync.exe "; //"%CCP_SDK%Bin\\mpicsync.exe ";
                                        string etl2otf = sdkpath + "Bin\\etl2otf.exe "; //"%CCP_SDK%Bin\\etl2otf.exe ";
                                        string etlsource = _dwarf.mpitracefile;

                                        glog.WriteLog("VIS", "Creating clock-sync data: " + mpicsync + etlsource);
                                        HelperMethods.ExecCommand(mpicsync, "\"" + etlsource + "\"", true);

                                        //Convert etl2otf
                                        glog.WriteLog("VIS", "Converting etl to otf: " + etl2otf + etlsource);
                                        HelperMethods.ExecCommand(etl2otf, "\"" + etlsource + "\"", true);

                                        //Convert clog2 to slog2
                                        glog.WriteLog("VIS", "Done converting trace files for dwarf " + _dwarf.dwarfname);

                                        otffile = etlsource.Replace(".etl", ".etl_otf.otf");

                                        glog.WriteLog("VIS", "Opening Vampir for: " + otffile);
                                        glog.WriteLog("VIS", "Vampir executable: " + gsettings.vampirbin);

                                        //launch vampir
                                        HelperMethods.ExecCommand(
                                            gsettings.vampirbin,
                                            "\"" + otffile + "\"",
                                            false);
                                    }
                                }
                                else
                                    glog.WriteLog("VIS", "Skipping Vampir for dwarf " + _dwarf.dwarfname + ", as mpi tracing was not enabled or dwarf failed.");
                            }
                        }

                    }
                    #endregion

                    #region PLOT_JUMPSHOT
                    if (_plotjumpshot)
                    {
                        if (!File.Exists(gsettings.jumpshotpath + "jumpshot.jar"))
                        {
                            glog.WriteLog("VIS", "Error, cannot access file: " + gsettings.jumpshotpath + "jumpshot.jar");
                            Console.WriteLine("Error: Cannot access JumpShot executable (" + gsettings.jumpshotpath + "jumpshot.jar" + ").");
                            Console.WriteLine("Cannot view results in JumpShot. Skipping \"-PlotJumpShot\"");
                        }
                        else
                        {

                            foreach (Dwarf2 _dwarf in dwarfs)
                            {
                                if (terminate)
                                    break;

                                if (_dwarf.success && _dwarf.mpitrace &&
                                    (
                                    _dwarf.parallel.Contains("mpi") ||
                                    _dwarf.parallel.Contains("mpinet") ||
                                    _dwarf.parallel.Contains("hybrid")
                                    )
                                    )
                                {
                                    glog.WriteLog("DB_WF", "Starting JumpShot visualisation");
                                    Console.WriteLine("Starting JumpShot visualisation.");
                                    glog.WriteLog("VIS", "Opening Jumpshot MPI trace analysis for " +
                                        _dwarf.dwarfname + "[" +
                                        _dwarf.platform + "," +
                                        _dwarf.parallel + "," +
                                        _dwarf.size + "]" + ".");

                                    string slog2file = "";
                                    //Process tracing information 
                                    string sdkpath = System.Environment.GetEnvironmentVariable("CCP_SDK");
                                    if (sdkpath != null) sdkpath = sdkpath.Trim(); else sdkpath = "";
                                    string mpicsync = sdkpath + "Bin\\mpicsync.exe "; //"%CCP_SDK%Bin\\mpicsync.exe ";
                                    string etl2clog = sdkpath + "Bin\\etl2clog.exe "; //"%CCP_SDK%Bin\\etl2otf.exe ";
                                    string etlsource = _dwarf.mpitracefile;


                                    if (runoncluster && (!String.IsNullOrEmpty(gsettings.hpcscheduler)))
                                    {
                                        glog.WriteLog("VIS", "Done converting trace files for dwarf " + _dwarf.dwarfname);
                                        glog.WriteLog("VIS", "Slog file: " + slog2file);


                                        //Convert clog2 to slog2
                                        string jumpshotdir = gsettings.jumpshotpath;
                                        string clog2slog = jumpshotdir + "clog2TOslog2.jar";
                                        string jumpshot = jumpshotdir + "jumpshot.jar";


                                        string javabin = gsettings.javabin;

                                        glog.WriteLog("VIS", "Converting clog2 to slog2: " + _dwarf.clogf);
                                        HelperMethods.ExecCommand(
                                            gsettings.javabin,
                                            " -jar " + clog2slog + " " + _dwarf.clogf,
                                            true);

                                        glog.WriteLog("VIS", "Opening Jumpshot for " + _dwarf.clogf.Replace("clog2", "slog2"));
                                        HelperMethods.ExecCommand(
                                            gsettings.javabin,
                                            " -jar " + jumpshot + " " + _dwarf.clogf.Replace("clog2", "slog2"),
                                            false);

                                    }
                                    else
                                    {

                                        //create cloc sync data
                                        glog.WriteLog("VIS", "Creating clock-sync data: " + mpicsync + etlsource);
                                        HelperMethods.ExecCommand(mpicsync, "\"" + etlsource + "\"", true);

                                        //Convert etl 2 otf
                                        glog.WriteLog("VIS", "Converting etl to clog: " + etl2clog + etlsource);
                                        HelperMethods.ExecCommand(etl2clog, "\"" + etlsource + "\"", true);

                                        //Convert clog2 to slog2
                                        string jumpshotdir = gsettings.jumpshotpath;
                                        string clog2slog = jumpshotdir + "clog2TOslog2.jar";
                                        string jumpshot = jumpshotdir + "jumpshot.jar";

                                        string clog2file = etlsource.Replace(".etl", ".etl.clog2");
                                        slog2file = clog2file.Replace(".clog2", ".slog2");

                                        string javabin = gsettings.javabin;

                                        glog.WriteLog("VIS", "Converting clog2 to slog2: " + clog2file);
                                        HelperMethods.ExecCommand(javabin, " -jar " + "\"" + clog2slog + "\" " + "\"" + clog2file + "\"", true);

                                        glog.WriteLog("VIS", "Opening Jumpshot for " + slog2file);
                                        HelperMethods.ExecCommand(javabin, " -jar " + "\"" + jumpshot + "\" " + "\"" + slog2file + "\"", false);

                                        glog.WriteLog("VIS", "Done converting trace files for dwarf " + _dwarf.dwarfname);

                                    }
                                }
                                else
                                    glog.WriteLog("VIS", "Skipping JumpShot for dwarf " + _dwarf.dwarfname + ", as mpi tracing was not enabled or dwarf failed.");
                            }
                        }
                    }
                    #endregion

                }
                else
                    glog.WriteLog("VIS", "Skipping result visualisation because of ctrl-c event");

            }
            catch (Exception exp)
            {
                Console.WriteLine("DwarfBench: General exception occured.");
                Console.WriteLine("Exception details: " + exp.Message);
                Console.WriteLine("Aborting execution.");
                glog.WriteLog("**EXCEPTION**", "Message: " + exp.Message);
                glog.WriteLog("**EXCEPTION**", "Inner exception: " + exp.InnerException);
                glog.WriteLog("**EXCEPTION**", "Source: " + exp.Source);
                glog.WriteLog("**EXCEPTION**", "Target: " + exp.TargetSite);
                glog.WriteLog("**EXCEPTION**", "Stack trace: " + exp.StackTrace);
                glog.WriteLog("DB_WF", "Aborting execution");

            }

            //glog.WriteLog("DB_WB", "Unregistring ctrl-c event handler");
            //Console.CancelKeyPress -= ConsoleEventHandler;

            whatif = false;
            help = false;
            isalive = false;
            glog.Close();
        }

        protected override void StopProcessing()
        {

            if (isalive)
            {
                terminate = true;
                glog.WriteLog("CMD_EVNT_HNDLR", "Ctrl-C detected.");
                Console.WriteLine("\nCtrl-C pressed. Terminating execution.");

                if (String.IsNullOrEmpty(gsettings.hpcscheduler))
                {
                    if (DwarfProcess != null)
                    {
                        try
                        {
                            DwarfProcess.Refresh();
                            if (!DwarfProcess.HasExited)
                            {
                                glog.WriteLog("CMD_EVNT_HNDLR", "Ctrl-C closing process with Id: " + DwarfProcess.Id);
                                DwarfProcess.Kill();
                                DwarfProcess.WaitForExit();
                                glog.WriteLog("CMD_EVNT_HNDLR", "Ctrl-C closed process with Id: " + DwarfProcess.Id);

                                if (xperfon)
                                {
                                    glog.WriteLog("CMD_EVNT_HNDLR", "Closing Xperf because of Ctrl-C signal.");
                                    HelperMethods.StopXperf();
                                }
                            }
                            else
                            {
                                glog.WriteLog("CMD_EVNT_HNDLR", "Tried to kill process that has exited.");
                            }
                        }
                        catch (Exception exp)
                        {
                            if (!exp.Message.ToString().Contains(" has exited.") &&
                                !exp.Message.ToString().Contains("No process is associated with this object"))
                            {
                                glog.WriteLog("**EXCEPTION**", "Error killing dwarf-process: " + exp.Message);
                                Console.WriteLine("Error: There seems to be aproblem with one of the running processes. ");
                                Console.WriteLine(exp.Message);

                                glog.WriteLog("**EXCEPTION**", "Inner exception: " + exp.InnerException);
                                glog.WriteLog("**EXCEPTION**", "Source: " + exp.Source);
                                glog.WriteLog("**EXCEPTION**", "Target: " + exp.TargetSite);
                                glog.WriteLog("**EXCEPTION**", "Stack trace: " + exp.StackTrace);
                            }
                            else
                                glog.WriteLog("CMD_EVNT_GNDLR", "Cannot kill current process as it has either exited or has not been started yet.");
                        }

                    }
                    else
                        glog.WriteLog("CMD_EVNT_HNDLR", "DwarfProcess == null. Nothing to terminate.");
                }
                else
                {
                    if (WScheduler.CurrentClusterJob != null)
                    {
                        if (WScheduler.CurrentClusterJob.IsActive)
                        {
                            glog.WriteLog("CMD_EVNT_HNDLR", "Ctrl-C pressed. Trying to terminate cluster job.");
                            WScheduler.CurrentClusterJob.Cancel("User interrupted execution with a Ctrl-C");
                        }
                    }
                    else
                        glog.WriteLog("CMD_EVNT_HNDLR", "DwarfClusterJob == null. Nothing to terminate.");
                }

            }
        }




        private List<Dwarf2> GetDwarfs3()
        {
            glog.WriteLog("GetDwarfs", "Generating dwarf configuration collection.");
            List<Dwarf2> dwarfslist = new List<Dwarf2>();

            string dwarf_bin = gsettings.dwarfdir + "Bin"; //@"E:\MY_SOURCES\DWARFS SRC_MAIN_SVN\Bin";

            DirectoryInfo root_dir = new DirectoryInfo(dwarf_bin);
            DirectoryInfo[] subdirs = root_dir.GetDirectories("Dwarf.*");

            foreach (DirectoryInfo dwarfdir in subdirs)
            {
                glog.WriteLog("GetDwarfs", "Analyzing " + dwarfdir);
                FileInfo[] binfiles = dwarfdir.GetFiles("Dwarf.*.exe");
                FilterFiles(ref binfiles);
                DirectoryInfo inputdir = new DirectoryInfo(dwarfdir.FullName + "\\Input");
                FileInfo[] inputfiles = inputdir.GetFiles();

                foreach (FileInfo dwarfbinary in binfiles)
                {
                    foreach (FileInfo inputfile in inputfiles)
                    {

                        //string exe = "\\" + "Dwarf." + _name.ToString() + "." + _platform.ToString() + "." + __parallel.ToString() + ".exe";
                        string exe = dwarfbinary.ToString();

                        string bindir = "";
                        string resultf = "";
                        string profilef = "";
                        string inputf = "";
                        string tracedir = "";


                        if (!String.IsNullOrEmpty(gsettings.hpcscheduler) && runoncluster)
                        {
                            if (dwarfbinary.ToString().ToLower().Contains("mpi") ||
                                dwarfbinary.ToString().ToLower().Contains("hybrid"))
                            {
                                bindir = gsettings.nodelocaltargetdir; //the binary is copied onto each node. So execution is local.
                                inputf = gsettings.sdeploymentdir + dwarfdir.Name + gsettings.inputdirectory + "\\";
                                resultf = gsettings.sdeploymentdir + dwarfdir.Name + "\\" + gsettings.resultdirectory + "\\" + dwarfbinary.Name.Replace("exe", inputfile.Name.ToString().Replace(".txt", "") + ".Result.txt");
                                profilef = gsettings.sdeploymentdir + dwarfdir.Name + "\\" + gsettings.profiledirectory + "\\" + dwarfbinary.Name.Replace("exe", inputfile.Name.ToString().Replace(".txt", "") + ".Profile.txt");
                                tracedir = gsettings.sdeploymentdir + dwarfdir.Name + "\\" + gsettings.profiledirectory + "\\";
                            }
                            else
                            {
                                bindir = gsettings.deploymentdir + "Bin\\" + dwarfdir.Name + "\\";
                                inputf = bindir + "\\" + gsettings.inputdirectory + "\\";
                                resultf = bindir + "\\" + gsettings.resultdirectory + "\\" + dwarfbinary.Name.Replace("exe", inputfile.Name.ToString().Replace(".txt", "") + ".Result.txt"); ;
                                profilef = bindir + "\\" + gsettings.profiledirectory + "\\" + dwarfbinary.Name.Replace("exe", inputfile.Name.ToString().Replace(".txt", "") + ".Profile.txt"); ;
                                tracedir = bindir + "\\" + gsettings.profiledirectory + "\\";
                            }

                        }
                        else
                        {
                            bindir = gsettings.deploymentdir + "Bin\\" + dwarfdir.Name + "\\";
                            inputf = bindir + "" + gsettings.inputdirectory + "\\";
                            resultf = bindir + "" + gsettings.resultdirectory + "\\" + dwarfbinary.Name.Replace("exe", inputfile.Name.ToString().Replace(".txt", "") + ".Result.txt"); ;
                            profilef = bindir + "" + gsettings.profiledirectory + "\\" + dwarfbinary.Name.Replace("exe", inputfile.Name.ToString().Replace(".txt", "") + ".Profile.txt"); ;
                            tracedir = bindir + "" + gsettings.profiledirectory + "\\";
                        }

                        inputf += inputfile.Name;

                        //override xml config for mpi
                        if ((override_proccount != gsettings.mpiproccount) && (override_proccount > 0))
                        {
                            glog.WriteLog("GET_DWARFS3", "Overriding gsettings mpiproccount with value: " + override_proccount);
                            gsettings.mpiproccount = override_proccount;
                            gsettings.mpiexecstr = gsettings.mpiexecstr.Replace("%PROCCOUNT%", gsettings.mpiproccount.ToString());
                        }
                        else if (gsettings.mpiproccount >= 0)
                            gsettings.mpiexecstr = gsettings.mpiexecstr.Replace("%PROCCOUNT%", gsettings.mpiproccount.ToString());
                        else
                        {
                            glog.WriteLog("GET_DWARFS3", "No mpi process count was specified. Setting default of 2.");
                            gsettings.mpiproccount = 2;
                            gsettings.mpiexecstr = "";
                        }

                        if (runoncluster && !String.IsNullOrEmpty(gsettings.hpcscheduler))
                        {
                            gsettings.mpiexecstr = "mpiexec.exe";
                        }
                        tracedir = bindir + gsettings.profiledirectory + "\\";

                        //override xml config for omp
                        if ((override_threadcount != gsettings.threadcount) && (override_threadcount > 1))
                        {
                            //glog.WriteLog("GET_DWARFS2", "Overriding gsettings threadcount with value: " + override_threadcount);
                            gsettings.threadcount = override_threadcount;
                        }

                        if (override_xperftrace)
                        {
                            //glog.WriteLog("GET_DWARFS2", "Xperf tracing enabled through Cli. Overriding dwarfconfig setting.");
                            gsettings.xperftracingenabled = true;
                        }

                        if (override_mpitrace)
                        {
                            //glog.WriteLog("GET_DWARFS2", "MPI tracing enabled through Cli. Overriding dwarfconfig setting.");
                            gsettings.mpitracingenabled = true;
                        }

                        string[] types = dwarfbinary.Name.Split('.');

                        string name = types[1];
                        string platform = types[2];
                        string parallel = types[3];
                        string size = "";
                        try
                        {
                            size = inputfile.Name.ToString().Replace(".txt", "");
                        }
                        catch (Exception)
                        {
                            size = inputfile.Name;
                        }


                        Dwarf2 currentdwarf = new Dwarf2(
                            name,
                            platform.ToLower(),
                            parallel.ToLower(),
                            size.ToLower(),
                            bindir + exe,
                            "", //args
                            bindir, //workdir
                            inputf,
                            resultf,
                            profilef,
                            gsettings.mpitracingenabled,
                            gsettings.xperftracingenabled,
                            gsettings.mpiexecstr,
                            gsettings.hpcscheduler,
                            "",
                            gsettings.threadcount,
                            gsettings.mpiproccount);

                        //set helper vars
                        currentdwarf.resultdir = gsettings.resultdirectory;
                        currentdwarf.profiledir = gsettings.profiledirectory;
                        currentdwarf.mpitracedir = tracedir;
                        currentdwarf._gsettings = gsettings;
                        currentdwarf.lmpitracedir = gsettings.sdeploymentdir + dwarfdir.Name + gsettings.profiledirectory + "\\";

                        //if (!String.IsNullOrEmpty(gsettings.hpcscheduler) &&
                        //    runonclusrter &&
                        //    (dwarfbinary.Name.Contains("mpi") || dwarfbinary.Name.Contains("hybrid")))
                        //{
                        //    currentdwarf.mpitracefile = ""; //fix to exclude 
                        //}

                        dwarfslist.Add(currentdwarf);
                    }
                }
            }
            return dwarfslist;
        }

        private void FilterFiles(ref FileInfo[] binfiles)
        {
            if (binfiles == null)
            {
                return;
            }

            List<FileInfo> tmp = new List<FileInfo>();
            foreach (FileInfo file in binfiles)
            {
                if (!file.Name.ToLower().Contains("vshost"))
                {
                    tmp.Add(file);
                }
                else
                {
                    glog.WriteLog("FILTER", "Removing file " + file.Name);
                }
            }

            binfiles = tmp.ToArray<FileInfo>();

        }


        private bool PassExclude(Dwarf2 dwarf)
        {
            if (dwarf == null)
                return true;
            if (gsettings == null)
                return true;

            string[] excludes = null;

            try
            {
                excludes = gsettings.excludelist.Split(';');
            }
            catch (Exception exp)
            {
                glog.WriteLog("GET_DWARFS2", "Cannot parse exclude list: " + exp.Message);
                return true;
            }

            if (excludes.Length == 0)
            {
                glog.WriteLog("GET_DWARFS2", "No exclude dwarf names specified.");
                return true;
            }

            string fulldwarfqualifier =
                dwarf.internalname.ToString() + "." +
                dwarf.platform.ToString() + "." +
                dwarf.parallel.ToString();

            fulldwarfqualifier = fulldwarfqualifier.ToLower();

            foreach (String excludename in excludes)
            {
                if (String.Compare(
                    fulldwarfqualifier,
                    excludename.ToLower()) == 0)
                {
                    glog.WriteLog("GET_DWARFS2", "Excluded dwarf " + fulldwarfqualifier);
                    return false;
                }
            }

            return true;

        }

        private bool PassFilter(Dwarf2 dwarf)
        {
            bool passed = false;

            if (filternames != null)
            {
                passed = false;
                foreach (string name in filternames)
                {
                    if (dwarf.internalname.ToLower().Equals(name.ToLower()))
                        passed = true;
                }
                if (!passed)
                    return false;
            }

            if (filterplatform != null)
            {
                passed = false;
                foreach (string platform in filterplatform)
                {
                    if (dwarf.platform.ToLower().Equals(platform.ToLower()))
                        passed = true;
                }
                if (!passed)
                    return false;
            }

            if (filterparallel != null)
            {
                passed = false;
                foreach (string parallel in filterparallel)
                {
                    if (dwarf.parallel.ToLower().Equals(parallel.ToLower()))
                        passed = true;
                }
                if (!passed)
                    return false;
            }

            if (filtersizes != null)
            {
                passed = false;
                foreach (string size in filtersizes)
                {
                    if (dwarf.size.ToLower().Contains(size.ToLower()))
                        passed = true;
                }
                if (!passed)
                    return false;
            }

            return true;
        }

        public void DisplayHelpMessage()
        {
            Console.WriteLine("Exec-Dwarfs");
            Console.WriteLine("         -names:         list of dwarf names");
            Console.WriteLine("         -platform:      specify the platform to execute (managed/unmanaged)");
            Console.WriteLine("         -parallel:      specify parallelization (serial/omp/mpi/tpl/hybrid)");
            Console.WriteLine("         -input:         specify input file name (small/medium/large)");
            Console.WriteLine("");
            Console.WriteLine("         -help:          displays help message");
            Console.WriteLine("         -whatif:        displays configuration but skips execution");
            Console.WriteLine("         -threadcount:   optional; specify number of OpenMP threads");
            Console.WriteLine("         -mpiproccount:  optional; specify number of MPI processes");
            Console.WriteLine("");
            Console.WriteLine("         -xperftrace:    which parameter enabling xperf tracing");
            Console.WriteLine("         -mpitrace:      switch parameter enabling mpi tracing");
            Console.WriteLine("         -cluster:       starts mpi or hybrid kernel on a Microsoft HPC cluster");
            Console.WriteLine("         -askedresource: specify level of resource allocation (node, socker or core");
            Console.WriteLine("");
            Console.WriteLine("Usage example: ");
            Console.WriteLine("Exec-Dwarfs -names structuredgrid -platform unmanaged -parallel mpi -input small");
            Console.WriteLine("Exec-Dwarfs -names structuredgrid -platform unmanaged -parallel mpi -input small -MpiTrace -PlotVampir");
            Console.WriteLine("Exec-Dwarfs -names structuredgrid -platform unmanaged -parallel mpi -input small -MpiTrace -PlotJumpShot");
            Console.WriteLine("Exec-Dwarfs -names structuredgrid -platform unmanaged -parallel mpi -input small -MpiTrace -PlotVampir -Cluster");
            Console.WriteLine("Exec-Dwarfs -names structuredgrid -platform unmanaged -parallel mpi -input large -Cluster -AskedResource node");
            Console.WriteLine("Exec-Dwarfs -names structuredgrid -platform unmanaged -parallel mpi -input small -XperfTrace -PlotXperf -PlotExcel");
            Console.WriteLine("");
            Console.WriteLine("Supported Dwarf names:");
            Console.WriteLine("BranchAndBound, CombinationalLogic, DenseAlgebra, DynamicProgramming, FiniteStateMachine, GraphModels, GraphTraversal, MapReduce,SparseAlgebra, SpectralMethod, StructuredGrid and UnstructuredGrid");

        }



        //EXECUTION
        private ProgressRecord all = null;
        private bool terminate = false;
        public static bool successfuljob = false;
        public static bool waitfornext = true;
        private static System.Diagnostics.Process DwarfProcess = null;
        private static bool xperfon = false;


        public int RunDwarfs(List<Dwarf2> dwarflist)
        {
            int failed = 0;
            double totalTime = 0;

            int dall = (int)(100 / dwarflist.ToArray().Length);
            this.WriteProgress(all);

            glog.WriteLog("RUN_DWARFS", "Starting execution of dwarflist.");
            Console.WriteLine("Starting execution.");
            string failedstring = "";

            foreach (Dwarf2 _dwarf in dwarflist)
            {
                glog.WriteLog("RUN_DWARFS", "Executing dwarf: " + _dwarf.internalname);
                if (override_mpitrace)
                {
                    glog.WriteLog("RUN_DWARFS", "Mpi tracing enabled through Cli. Overriding dwarfconfig setting.");
                    _dwarf.mpitrace = override_mpitrace;
                }

                if (!terminate)
                {
                    bool passedconsistency = false;
                    failedstring = "";

                    //skip consistency test for cluster execution as files have not been deployed to cluster at this point
                    if ((_dwarf.parallel.Contains("mpi") ||
                        _dwarf.parallel.Contains("hybrid")) &&
                        (!String.IsNullOrEmpty(gsettings.hpcscheduler)) &&
                        runoncluster)
                    {
                        glog.WriteLog("RUN_DWARFS", "Skipping consistency test for cluster execution.");
                        passedconsistency = true;
                    }
                    else
                        passedconsistency = _dwarf.CeckConsistency(glog, ref failedstring);

                    Console.Write(_dwarf.dwarfname + "." + _dwarf.platform + "." + _dwarf.parallel + "." + _dwarf.size + " started...");
                    if (passedconsistency)
                    {
                        string execmd = _dwarf.ConstructExecCommand();
                        totalTime = ExecuteDwarf(_dwarf, ref failedstring);
                    }
                    else
                    {
                        _dwarf.success = false;
                        glog.WriteLog("RUN_DWARFS", "Dwarf does not meet consistency criteria. Skipping. ");
                    }

                    if (!terminate)
                    {
                        all.PercentComplete += dall;
                        this.WriteProgress(all);
                    }
                }
                else
                {
                    _dwarf.success = false;
                    _dwarf.errormsg = "Execution skipped because of Run-Dwarf termination";
                    glog.WriteLog("RUN_DWARFS", "Skipping dwarf execution because Ctrl-C (terminate == true).");
                }

                if (_dwarf.success)
                {
                    try
                    {
                        string resultserror = "";
                        bool successfulread = _dwarf.GetResult(ref resultserror);
                        if (successfulread)
                        {
                            if (!String.IsNullOrEmpty(resultserror))
                            {
                                Console.WriteLine(" done. Error: " + resultserror);
                            }
                            else
                            {
                                Console.WriteLine(" done. Error occured processing results.");
                            }

                        }
                        else
                        {
                            Console.WriteLine(" done. Dwarf kernel time: " + _dwarf.time + " sec.");
                            glog.WriteLog("RUN_DWARFS", "Total processor time: " + totalTime + " sec.");
                        }
                    }
                    catch (Exception getresultexp)
                    {
                        Console.WriteLine(" done. Kernel time: FAILED reading profile file.");
                        glog.WriteLog("RUN_DWARFS", "Total processor time: FAILED reading profile file.");
                        glog.WriteLog("RUN_DWARFS", "Profile file reading exception: " + getresultexp.Message);
                        glog.WriteLog("RUN_DWARFS", "Setting dwarf to failed");
                        _dwarf.success = false;
                    }
                }
                else
                {
                    if (!terminate)
                    {
                        if (!String.IsNullOrEmpty(failedstring))
                            Console.Write("(" + failedstring + ")");
                        Console.WriteLine(" failed.");
                    }
                    failed++;
                }

            }

            if (!terminate)
            {
                all.PercentComplete = 100;
                this.WriteProgress(all);
            }

            return failed;
        }


        private double ExecuteDwarf(Dwarf2 dwarf, ref string failedstring)
        {

            System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
            DwarfProcess = new System.Diagnostics.Process();

            //Cluster execution
            if ((dwarf.parallel.Contains("mpi") ||
                dwarf.parallel.Contains("hybrid")) &&
                (!String.IsNullOrEmpty(gsettings.hpcscheduler)) &&
                runoncluster)
            {
                #region CLUSTER_EXEC
                try
                {
                    WScheduler Cluster = new WScheduler(glog, gsettings, jobFinishEvt, _plotvampir, _plotjumpshot);
                    failedstring = Cluster.ClusterExec(dwarf, failedstring, watch);
                }
                catch (Exception e)
                {
                    glog.WriteLog("EXECUTE_DWARF", "Exception creating Wcluster instance.");
                    glog.WriteLog("EXECUTE_DWARF", e.Message);
                    glog.WriteLog("EXECUTE_DWARF", e.StackTrace);
                    failedstring = "cluster execution failed";
                    Console.WriteLine("Cluster execution failed: Please make sure HPC CCP client tools are installed.");
                    dwarf.success = false;
                }
                return watch.ElapsedMilliseconds / (double)1000;

                #endregion
            }
            //Local execution
            else
            {
                #region LOCAL_EXEC

                if (dwarf.parallel.Contains("hybrid") ||
                    dwarf.parallel.Contains("mpi"))
                {
                    string mpiexecpath = System.Environment.GetEnvironmentVariable("CCP_HOME");

                    if (!String.IsNullOrEmpty(mpiexecpath))
                    {
                        mpiexecpath = mpiexecpath.Trim();
                        if (File.Exists(@mpiexecpath + "Bin\\mpiexec.exe"))
                            DwarfProcess.StartInfo.FileName = @mpiexecpath + "Bin\\mpiexec.exe";
                        else
                        {
                            mpiexecpath = "";
                        }
                    }

                    if (String.IsNullOrEmpty(mpiexecpath))
                    {
                        mpiexecpath = System.Environment.GetEnvironmentVariable("CCP_SDK");
                        if (!String.IsNullOrEmpty(mpiexecpath))
                        {
                            mpiexecpath = mpiexecpath.Trim();

                            if (File.Exists(@mpiexecpath + "Bin\\mpiexec.exe"))
                                DwarfProcess.StartInfo.FileName = @mpiexecpath + "Bin\\mpiexec.exe";
                            else
                            {
                                mpiexecpath = "";
                            }
                        }
                    }

                    if (String.IsNullOrEmpty(mpiexecpath))
                    {
                        glog.WriteLog("EXECUTE_DWARF", "Environment variables CCP_SDK or CCP_HOME not set.");
                        Console.WriteLine("Warning: Cannot find mpiexec.exe path. Please make sure Microsoft HPC pack (SDK) is installed.");
                        DwarfProcess.StartInfo.FileName = "mpiexec.exe";
                    }
                }
                else
                {
                    DwarfProcess.StartInfo.FileName = dwarf.bin;
                }


                DwarfProcess.StartInfo.WorkingDirectory = dwarf.workdir;
                DwarfProcess.StartInfo.Arguments = dwarf.ConstructExecCommand();
                DwarfProcess.StartInfo.UseShellExecute = false;
                DwarfProcess.StartInfo.CreateNoWindow = true;
                DwarfProcess.StartInfo.RedirectStandardOutput = true;
                DwarfProcess.StartInfo.RedirectStandardError = true;
                //Redirect streams
                DwarfProcess.OutputDataReceived += new System.Diagnostics.DataReceivedEventHandler(DwarfProcess_OutputDataReceived);
                DwarfProcess.ErrorDataReceived += new System.Diagnostics.DataReceivedEventHandler(DwarfProcess_ErrorDataReceived);

                glog.WriteLog("EXECUTE_DWARF", "DwarfProcess startinfo filename: " + DwarfProcess.StartInfo.FileName);
                glog.WriteLog("EXECUTE_DWARF", "DwarfProcess startinfo args: " + DwarfProcess.StartInfo.Arguments);
                glog.WriteLog("EXECUTE_DWARF", "DwarfProcess wdir: " + DwarfProcess.StartInfo.WorkingDirectory);

                if (dwarf.parallel.Contains("omp"))
                {
                    DwarfProcess.StartInfo.EnvironmentVariables.Add("OMP_NUM_THREADS", dwarf.numthreads.ToString());
                }

                if (dwarf.xperftrace)
                {
                    glog.WriteLog("EXECUTE_DWARF", "Xperf active with trace file: " + dwarf.xperftracefile);
                    xperfon = true;
                    HelperMethods.StartXperf(dwarf.xperftracefile);
                }

                glog.WriteLog("EXECUTE_DWARF", "Launching local process and waiting " + gsettings.proctimeout + " ms for completion.");

                watch.Start();
                DwarfProcess.Start();
                glog.WriteLog("EXECUTE_DWARF", "Process (Id: " + DwarfProcess.Id + ") has started.");

                DwarfProcess.BeginOutputReadLine();
                DwarfProcess.BeginErrorReadLine();

                DwarfProcess.WaitForExit(gsettings.proctimeout);

                watch.Stop();

                if (DwarfProcess != null)
                {
                    dwarf.log_process_execution_string +=
                        DwarfProcess.StartInfo.FileName + " " +
                        DwarfProcess.StartInfo.Arguments;

                    DwarfProcess.Refresh();
                    if (!DwarfProcess.HasExited)
                    {
                        glog.WriteLog("EXECUTE_DWARF", "Timeout closing process with Id: " + DwarfProcess.Id);
                        DwarfProcess.Kill();
                        DwarfProcess.WaitForExit();
                        glog.WriteLog("EXECUTE_DWARF", "Timeout closed process with Id: " + DwarfProcess.Id);
                        failedstring = "time out";
                    }
                }

                glog.WriteLog("EXECUTE_DWARF", "Process (Id: " + DwarfProcess.Id + ") has exited.\n");

                if (dwarf.xperftrace)
                {
                    HelperMethods.StopXperf();
                    xperfon = false;
                    glog.WriteLog("EXECUTE_DWARF", "Xperf stopped.");
                }


                DwarfProcess.Refresh();
                int exitcode = DwarfProcess.ExitCode;
                glog.WriteLog("EXECUTE_DWARF", "Process (Id: " + DwarfProcess.Id + ") has exited with ExitCode " + exitcode + ".");

                if (exitcode != 0)
                    dwarf.success = false;
                else
                    dwarf.success = true;
                #endregion
            }

            return watch.ElapsedMilliseconds / (double)1000;
        }

        //EVENTS
        private ManualResetEvent jobFinishEvt = new ManualResetEvent(false);


        private void SessionEndHandler(object o, Microsoft.Win32.SessionEndingEventArgs args)
        {
            Console.WriteLine("Ending");
            Console.ReadKey();
        }


        void DwarfProcess_ErrorDataReceived(object sender, System.Diagnostics.DataReceivedEventArgs e)
        {
            if (!String.IsNullOrEmpty(e.Data))
            {
                glog.WriteLog("STD_ERR", e.Data);
            }
        }


        void DwarfProcess_OutputDataReceived(object sender, System.Diagnostics.DataReceivedEventArgs e)
        {
            if (!String.IsNullOrEmpty(e.Data))
            {
                glog.WriteLog("STD_OUT", e.Data);
            }
        }



    }
}
